#!/bin/bash
num_nodes=2
node_id=$1

SCRIPT_PATH=/share/projset/dsir7
cd $SCRIPT_PATH
CONFIG_FILE=/share/projset/dsir7/indexes/qwen_tokenizer_3_300000/feat_config.json
NUM_TO_SAMPLE=4200000
MIN_EXAMPLE_LENGTH=0

total_lines=$(wc -l < /share/projset/dsir7/scripts/fineweb-ed4u.txt)

# 遍历文件的行
for ((i=node_id; i<total_lines;i=i+num_nodes))
do
  # 计算当前行号
  current_line=$((i+1))
  
  # 获取当前行的内容
  filename=$(sed -n "${current_line}p" /share/projset/dsir7/scripts/fineweb-ed4u.txt)
  
  basename=$(echo $filename | rev | cut -d'-' -f2- | rev)
  version=$(echo $filename | sed -n 's/.*llmeval\([a-zA-Z0-9]*\)\.jsonl/\1/p')
  RAW_DATASETS=/share/projset/fineweb-edu/sample/jsons/"$basename".jsonl
  TARGET_DATASETS=/share/projset/dsir7/data/llm-eval-"$version"-merge.jsonl
  MERGED_DIR=/share/projset/dsir7/indexes/qwen_tokenizer_3_300000/"$basename"_llm-eval-"$version"-merge_0.0_0/resampled
  OUTFILE=/share/projset/dsir7/indexes/qwen_tokenizer_3_300000/"$basename"-llmeval"$version".jsonl
  # 在这里处理result变量，例如打印出来
  bash scripts/search.sh $CONFIG_FILE $RAW_DATASETS $TARGET_DATASETS $NUM_TO_SAMPLE $MIN_EXAMPLE_LENGTH
  bash scripts/merged_jsonl.sh $MERGED_DIR $OUTFILE
  wait
  echo "Node $node_id $i: $result $base_name $version"
  echo $RAW_DATASETS
  echo $TARGET_DATASETS
  echo $MERGED_DIR
  echo $OUTFILE
  echo "FINISHED"
done
